{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 使用submitter对api进行访问\n",
    "from sparksampling import Submitter\n",
    "from sparksampling.utilities.var import FILE_TYPE_CSV\n",
    "from sparksampling.utilities.var import SIMPLE_RANDOM_SAMPLING_METHOD\n",
    "submitter = Submitter()\n",
    "dataset_uri = 'hdfs://localhost:9000/dataset/titanic/train.csv'\n",
    "fraction = 0.4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "request: http://localhost:8000/v1/sampling/simplejob/\n",
      "2021-04-01 19:32:00,714 DEBUG connectionpool.py[231] Starting new HTTP connection (1): localhost:8000\n",
      "2021-04-01 19:32:00,745 DEBUG connectionpool.py[461] http://localhost:8000 \"POST /v1/sampling/simplejob/ HTTP/1.1\" 200 49\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'code': 0, 'msg': '', 'data': {'job_id': 10050}}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 提交抽样任务\n",
    "submit_response = submitter.submit_sampling_simplejob(dataset_uri,\n",
    "                                              method=SIMPLE_RANDOM_SAMPLING_METHOD,\n",
    "                                              file_type=FILE_TYPE_CSV,\n",
    "                                              fraction=fraction,\n",
    "                                              with_header=True)\n",
    "job_id = submit_response.job_id\n",
    "submit_response.to_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "request: http://localhost:8000/v1/sampling/query/job/\n",
      "2021-04-01 19:32:10,410 DEBUG connectionpool.py[231] Starting new HTTP connection (1): localhost:8000\n",
      "2021-04-01 19:32:10,416 DEBUG connectionpool.py[461] http://localhost:8000 \"POST /v1/sampling/query/job/ HTTP/1.1\" 200 462\n",
      "{'code': 0, 'msg': '', 'data': {'job_id': 10050, 'job_status': 'Succeed', 'msg': 'succeed', 'method': 'Simple Random Sampling', 'start_time': '2021/04/01/ 19:32:01', 'end_time': '2021/04/01 19:32:02', 'simpled_file_path': 'hdfs://localhost:9000/dataset/titanic/train.csv-sampled-1617276720.7170603', 'request_data': \"{'path': 'hdfs://localhost:9000/dataset/titanic/train.csv', 'method': 'random', 'type': 'csv', 'with_header': True, 'conf': {'fraction': 0.4}}\"}}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'hdfs://localhost:9000/dataset/titanic/train.csv-sampled-1617276720.7170603'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 查询抽样任务\n",
    "sampling_job_details = submitter.get_sampling_job_details(job_id)\n",
    "sampled_path = sampling_job_details.sampled_path\n",
    "print(sampling_job_details.to_dict())\n",
    "sampled_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PassengerId</th>\n",
       "      <th>Survived</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Name</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Ticket</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Cabin</th>\n",
       "      <th>Embarked</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Cumings, Mrs. John Bradley (Florence Briggs Th...</td>\n",
       "      <td>female</td>\n",
       "      <td>38</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>PC 17599</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>C85</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Cumings, Mrs. John Bradley (Florence Briggs Th...</td>\n",
       "      <td>female</td>\n",
       "      <td>38</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>PC 17599</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>C85</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>Heikkinen, Miss. Laina</td>\n",
       "      <td>female</td>\n",
       "      <td>26</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>STON/O2. 3101282</td>\n",
       "      <td>7.925</td>\n",
       "      <td>None</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Allen, Mr. William Henry</td>\n",
       "      <td>male</td>\n",
       "      <td>35</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>373450</td>\n",
       "      <td>8.05</td>\n",
       "      <td>None</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Palsson, Master. Gosta Leonard</td>\n",
       "      <td>male</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>349909</td>\n",
       "      <td>21.075</td>\n",
       "      <td>None</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>371</th>\n",
       "      <td>881</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>Shelley, Mrs. William (Imanita Parrish Hall)</td>\n",
       "      <td>female</td>\n",
       "      <td>25</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>230433</td>\n",
       "      <td>26</td>\n",
       "      <td>None</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>372</th>\n",
       "      <td>885</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Sutehall, Mr. Henry Jr</td>\n",
       "      <td>male</td>\n",
       "      <td>25</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>SOTON/OQ 392076</td>\n",
       "      <td>7.05</td>\n",
       "      <td>None</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>373</th>\n",
       "      <td>886</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Rice, Mrs. William (Margaret Norton)</td>\n",
       "      <td>female</td>\n",
       "      <td>39</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>382652</td>\n",
       "      <td>29.125</td>\n",
       "      <td>None</td>\n",
       "      <td>Q</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>374</th>\n",
       "      <td>889</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>\"Johnston, Miss. Catherine Helen \"\"Carrie\"\"\"</td>\n",
       "      <td>female</td>\n",
       "      <td>None</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>W./C. 6607</td>\n",
       "      <td>23.45</td>\n",
       "      <td>None</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>375</th>\n",
       "      <td>889</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>\"Johnston, Miss. Catherine Helen \"\"Carrie\"\"\"</td>\n",
       "      <td>female</td>\n",
       "      <td>None</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>W./C. 6607</td>\n",
       "      <td>23.45</td>\n",
       "      <td>None</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>376 rows × 12 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    PassengerId Survived Pclass  \\\n",
       "0             2        1      1   \n",
       "1             2        1      1   \n",
       "2             3        1      3   \n",
       "3             5        0      3   \n",
       "4             8        0      3   \n",
       "..          ...      ...    ...   \n",
       "371         881        1      2   \n",
       "372         885        0      3   \n",
       "373         886        0      3   \n",
       "374         889        0      3   \n",
       "375         889        0      3   \n",
       "\n",
       "                                                  Name     Sex   Age SibSp  \\\n",
       "0    Cumings, Mrs. John Bradley (Florence Briggs Th...  female    38     1   \n",
       "1    Cumings, Mrs. John Bradley (Florence Briggs Th...  female    38     1   \n",
       "2                               Heikkinen, Miss. Laina  female    26     0   \n",
       "3                             Allen, Mr. William Henry    male    35     0   \n",
       "4                       Palsson, Master. Gosta Leonard    male     2     3   \n",
       "..                                                 ...     ...   ...   ...   \n",
       "371       Shelley, Mrs. William (Imanita Parrish Hall)  female    25     0   \n",
       "372                             Sutehall, Mr. Henry Jr    male    25     0   \n",
       "373               Rice, Mrs. William (Margaret Norton)  female    39     0   \n",
       "374       \"Johnston, Miss. Catherine Helen \"\"Carrie\"\"\"  female  None     1   \n",
       "375       \"Johnston, Miss. Catherine Helen \"\"Carrie\"\"\"  female  None     1   \n",
       "\n",
       "    Parch            Ticket     Fare Cabin Embarked  \n",
       "0       0          PC 17599  71.2833   C85        C  \n",
       "1       0          PC 17599  71.2833   C85        C  \n",
       "2       0  STON/O2. 3101282    7.925  None        S  \n",
       "3       0            373450     8.05  None        S  \n",
       "4       1            349909   21.075  None        S  \n",
       "..    ...               ...      ...   ...      ...  \n",
       "371     1            230433       26  None        S  \n",
       "372     0   SOTON/OQ 392076     7.05  None        S  \n",
       "373     5            382652   29.125  None        Q  \n",
       "374     2        W./C. 6607    23.45  None        S  \n",
       "375     2        W./C. 6607    23.45  None        S  \n",
       "\n",
       "[376 rows x 12 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 简单的读取抽样之后的文件\n",
    "from pyspark.sql import SparkSession\n",
    "from sparksampling.config import SPARK_CONF\n",
    "\n",
    "conf = SPARK_CONF\n",
    "spark = SparkSession.builder.config(conf=conf).getOrCreate()\n",
    "df = spark.read.csv(sampled_path, header=True)\n",
    "df.toPandas()\n",
    "# 可以在这后面做数据分析，或试试看下面的统计、评估功能\n",
    "\n",
    "# data analyse here\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "request: http://localhost:8000/v1/evaluation/statistics/\n",
      "2021-04-01 19:33:52,601 DEBUG connectionpool.py[231] Starting new HTTP connection (1): localhost:8000\n",
      "2021-04-01 19:33:53,704 DEBUG connectionpool.py[461] http://localhost:8000 \"POST /v1/evaluation/statistics/ HTTP/1.1\" 200 1414\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PassengerId</th>\n",
       "      <th>Survived</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Name</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Ticket</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Cabin</th>\n",
       "      <th>Embarked</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>summary</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>891</td>\n",
       "      <td>891</td>\n",
       "      <td>891</td>\n",
       "      <td>891</td>\n",
       "      <td>891</td>\n",
       "      <td>714</td>\n",
       "      <td>891</td>\n",
       "      <td>891</td>\n",
       "      <td>891</td>\n",
       "      <td>891</td>\n",
       "      <td>204</td>\n",
       "      <td>889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>446.0</td>\n",
       "      <td>0.3838383838383838</td>\n",
       "      <td>2.308641975308642</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>29.69911764705882</td>\n",
       "      <td>0.5230078563411896</td>\n",
       "      <td>0.38159371492704824</td>\n",
       "      <td>260318.54916792738</td>\n",
       "      <td>32.2042079685746</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>stddev</th>\n",
       "      <td>257.3538420152301</td>\n",
       "      <td>0.48659245426485753</td>\n",
       "      <td>0.8360712409770491</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>14.526497332334035</td>\n",
       "      <td>1.1027434322934315</td>\n",
       "      <td>0.8060572211299488</td>\n",
       "      <td>471609.26868834975</td>\n",
       "      <td>49.69342859718089</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>\"Andersson, Mr. August Edvard (\"\"Wennerstrom\"\")\"</td>\n",
       "      <td>female</td>\n",
       "      <td>0.42</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>110152</td>\n",
       "      <td>0</td>\n",
       "      <td>A10</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>99</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>van Melkebeke, Mr. Philemon</td>\n",
       "      <td>male</td>\n",
       "      <td>9</td>\n",
       "      <td>8</td>\n",
       "      <td>6</td>\n",
       "      <td>WE/P 5735</td>\n",
       "      <td>93.5</td>\n",
       "      <td>T</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               PassengerId             Survived              Pclass  \\\n",
       "summary                                                               \n",
       "count                  891                  891                 891   \n",
       "mean                 446.0   0.3838383838383838   2.308641975308642   \n",
       "stddev   257.3538420152301  0.48659245426485753  0.8360712409770491   \n",
       "min                      1                    0                   1   \n",
       "max                     99                    1                   3   \n",
       "\n",
       "                                                     Name     Sex  \\\n",
       "summary                                                             \n",
       "count                                                 891     891   \n",
       "mean                                                 None    None   \n",
       "stddev                                               None    None   \n",
       "min      \"Andersson, Mr. August Edvard (\"\"Wennerstrom\"\")\"  female   \n",
       "max                           van Melkebeke, Mr. Philemon    male   \n",
       "\n",
       "                        Age               SibSp                Parch  \\\n",
       "summary                                                                \n",
       "count                   714                 891                  891   \n",
       "mean      29.69911764705882  0.5230078563411896  0.38159371492704824   \n",
       "stddev   14.526497332334035  1.1027434322934315   0.8060572211299488   \n",
       "min                    0.42                   0                    0   \n",
       "max                       9                   8                    6   \n",
       "\n",
       "                     Ticket               Fare Cabin Embarked  \n",
       "summary                                                        \n",
       "count                   891                891   204      889  \n",
       "mean     260318.54916792738   32.2042079685746  None     None  \n",
       "stddev   471609.26868834975  49.69342859718089  None     None  \n",
       "min                  110152                  0   A10        C  \n",
       "max               WE/P 5735               93.5     T        S  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 统计原数据集\n",
    "submitter.get_statistics(path=dataset_uri, file_type=FILE_TYPE_CSV,with_header=True).to_pandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "request: http://localhost:8000/v1/evaluation/statistics/\n",
      "2021-04-01 19:33:57,520 DEBUG connectionpool.py[231] Starting new HTTP connection (1): localhost:8000\n",
      "2021-04-01 19:33:57,965 DEBUG connectionpool.py[461] http://localhost:8000 \"POST /v1/evaluation/statistics/ HTTP/1.1\" 200 1414\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PassengerId</th>\n",
       "      <th>Survived</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Name</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Ticket</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Cabin</th>\n",
       "      <th>Embarked</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>summary</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>376</td>\n",
       "      <td>376</td>\n",
       "      <td>376</td>\n",
       "      <td>376</td>\n",
       "      <td>376</td>\n",
       "      <td>296</td>\n",
       "      <td>376</td>\n",
       "      <td>376</td>\n",
       "      <td>376</td>\n",
       "      <td>376</td>\n",
       "      <td>102</td>\n",
       "      <td>374</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>464.2925531914894</td>\n",
       "      <td>0.39095744680851063</td>\n",
       "      <td>2.247340425531915</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>29.603918918918918</td>\n",
       "      <td>0.5159574468085106</td>\n",
       "      <td>0.38563829787234044</td>\n",
       "      <td>262423.08921933087</td>\n",
       "      <td>34.09537765957447</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>stddev</th>\n",
       "      <td>261.3650413713812</td>\n",
       "      <td>0.4886150640852816</td>\n",
       "      <td>0.8702449316915051</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>15.458875203458867</td>\n",
       "      <td>1.0977604539171548</td>\n",
       "      <td>0.7883864480629801</td>\n",
       "      <td>454985.4185076666</td>\n",
       "      <td>45.357145181350575</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>\"Bowen, Mr. David John \"\"Dai\"\"\"</td>\n",
       "      <td>female</td>\n",
       "      <td>0.42</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>110152</td>\n",
       "      <td>0</td>\n",
       "      <td>A16</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>97</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>Williams, Mr. Charles Eugene</td>\n",
       "      <td>male</td>\n",
       "      <td>9</td>\n",
       "      <td>8</td>\n",
       "      <td>6</td>\n",
       "      <td>WE/P 5735</td>\n",
       "      <td>93.5</td>\n",
       "      <td>G6</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               PassengerId             Survived              Pclass  \\\n",
       "summary                                                               \n",
       "count                  376                  376                 376   \n",
       "mean     464.2925531914894  0.39095744680851063   2.247340425531915   \n",
       "stddev   261.3650413713812   0.4886150640852816  0.8702449316915051   \n",
       "min                     10                    0                   1   \n",
       "max                     97                    1                   3   \n",
       "\n",
       "                                    Name     Sex                 Age  \\\n",
       "summary                                                                \n",
       "count                                376     376                 296   \n",
       "mean                                None    None  29.603918918918918   \n",
       "stddev                              None    None  15.458875203458867   \n",
       "min      \"Bowen, Mr. David John \"\"Dai\"\"\"  female                0.42   \n",
       "max         Williams, Mr. Charles Eugene    male                   9   \n",
       "\n",
       "                      SibSp                Parch              Ticket  \\\n",
       "summary                                                                \n",
       "count                   376                  376                 376   \n",
       "mean     0.5159574468085106  0.38563829787234044  262423.08921933087   \n",
       "stddev   1.0977604539171548   0.7883864480629801   454985.4185076666   \n",
       "min                       0                    0              110152   \n",
       "max                       8                    6           WE/P 5735   \n",
       "\n",
       "                       Fare Cabin Embarked  \n",
       "summary                                     \n",
       "count                   376   102      374  \n",
       "mean      34.09537765957447  None     None  \n",
       "stddev   45.357145181350575  None     None  \n",
       "min                       0   A16        C  \n",
       "max                    93.5    G6        S  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 统计抽样后的数据集\n",
    "data = submitter.get_statistics(job_id=job_id, from_sampling=True, file_type=FILE_TYPE_CSV,with_header=True).to_pandas()\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "request: http://localhost:8000/v1/evaluation/job/\n",
      "2021-04-01 19:34:01,722 DEBUG connectionpool.py[231] Starting new HTTP connection (1): localhost:8000\n",
      "2021-04-01 19:34:01,743 DEBUG connectionpool.py[461] http://localhost:8000 \"POST /v1/evaluation/job/ HTTP/1.1\" 200 49\n",
      "{'code': 0, 'msg': '', 'data': {'job_id': 50023}}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "50023"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 提交评估任务\n",
    "evaluation_job = submitter.submit_evaluation_job(compare_job_id=job_id, file_type=FILE_TYPE_CSV)\n",
    "print(evaluation_job.to_dict())\n",
    "evaluation_job_id = evaluation_job.job_id\n",
    "evaluation_job_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "request: http://localhost:8000/v1/evaluation/query/job/\n",
      "2021-04-01 19:34:07,985 DEBUG connectionpool.py[231] Starting new HTTP connection (1): localhost:8000\n",
      "2021-04-01 19:34:07,990 DEBUG connectionpool.py[461] http://localhost:8000 \"POST /v1/evaluation/query/job/ HTTP/1.1\" 200 2610\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Age</th>\n",
       "      <th>Cabin</th>\n",
       "      <th>Embarked</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Name</th>\n",
       "      <th>Parch</th>\n",
       "      <th>PassengerId</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Sex</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Survived</th>\n",
       "      <th>Ticket</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>296</td>\n",
       "      <td>102</td>\n",
       "      <td>374</td>\n",
       "      <td>376</td>\n",
       "      <td>376</td>\n",
       "      <td>376</td>\n",
       "      <td>376</td>\n",
       "      <td>376</td>\n",
       "      <td>376</td>\n",
       "      <td>376</td>\n",
       "      <td>376</td>\n",
       "      <td>376</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>29.603918918918918</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>34.09537765957447</td>\n",
       "      <td>None</td>\n",
       "      <td>0.38563829787234044</td>\n",
       "      <td>464.2925531914894</td>\n",
       "      <td>2.247340425531915</td>\n",
       "      <td>None</td>\n",
       "      <td>0.5159574468085106</td>\n",
       "      <td>0.39095744680851063</td>\n",
       "      <td>262423.08921933087</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>stddev</th>\n",
       "      <td>15.458875203458867</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>45.357145181350575</td>\n",
       "      <td>None</td>\n",
       "      <td>0.7883864480629801</td>\n",
       "      <td>261.3650413713812</td>\n",
       "      <td>0.8702449316915051</td>\n",
       "      <td>None</td>\n",
       "      <td>1.0977604539171548</td>\n",
       "      <td>0.4886150640852816</td>\n",
       "      <td>454985.4185076666</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.42</td>\n",
       "      <td>A16</td>\n",
       "      <td>C</td>\n",
       "      <td>0</td>\n",
       "      <td>\"Bowen, Mr. David John \"\"Dai\"\"\"</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>female</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>110152</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>9</td>\n",
       "      <td>G6</td>\n",
       "      <td>S</td>\n",
       "      <td>93.5</td>\n",
       "      <td>Williams, Mr. Charles Eugene</td>\n",
       "      <td>6</td>\n",
       "      <td>97</td>\n",
       "      <td>3</td>\n",
       "      <td>male</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>WE/P 5735</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean_bias</th>\n",
       "      <td>0.00320544</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>0.0587243</td>\n",
       "      <td>-1</td>\n",
       "      <td>0.0105992</td>\n",
       "      <td>0.0410147</td>\n",
       "      <td>0.0265531</td>\n",
       "      <td>-1</td>\n",
       "      <td>0.0134805</td>\n",
       "      <td>0.018547</td>\n",
       "      <td>0.00808448</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>stddev_bias</th>\n",
       "      <td>0.0641846</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>0.0872607</td>\n",
       "      <td>-1</td>\n",
       "      <td>0.0219225</td>\n",
       "      <td>0.0155863</td>\n",
       "      <td>0.0408741</td>\n",
       "      <td>-1</td>\n",
       "      <td>0.00451871</td>\n",
       "      <td>0.00415668</td>\n",
       "      <td>0.0352492</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>score</th>\n",
       "      <td>96.6305</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>92.7007</td>\n",
       "      <td>-1</td>\n",
       "      <td>98.3739</td>\n",
       "      <td>97.1699</td>\n",
       "      <td>96.6286</td>\n",
       "      <td>-1</td>\n",
       "      <td>99.1</td>\n",
       "      <td>98.8648</td>\n",
       "      <td>97.8333</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            Age Cabin Embarked                Fare  \\\n",
       "count                       296   102      374                 376   \n",
       "mean         29.603918918918918  None     None   34.09537765957447   \n",
       "stddev       15.458875203458867  None     None  45.357145181350575   \n",
       "min                        0.42   A16        C                   0   \n",
       "max                           9    G6        S                93.5   \n",
       "mean_bias            0.00320544    -1       -1           0.0587243   \n",
       "stddev_bias           0.0641846    -1       -1           0.0872607   \n",
       "score                   96.6305    -1       -1             92.7007   \n",
       "\n",
       "                                        Name                Parch  \\\n",
       "count                                    376                  376   \n",
       "mean                                    None  0.38563829787234044   \n",
       "stddev                                  None   0.7883864480629801   \n",
       "min          \"Bowen, Mr. David John \"\"Dai\"\"\"                    0   \n",
       "max             Williams, Mr. Charles Eugene                    6   \n",
       "mean_bias                                 -1            0.0105992   \n",
       "stddev_bias                               -1            0.0219225   \n",
       "score                                     -1              98.3739   \n",
       "\n",
       "                   PassengerId              Pclass     Sex  \\\n",
       "count                      376                 376     376   \n",
       "mean         464.2925531914894   2.247340425531915    None   \n",
       "stddev       261.3650413713812  0.8702449316915051    None   \n",
       "min                         10                   1  female   \n",
       "max                         97                   3    male   \n",
       "mean_bias            0.0410147           0.0265531      -1   \n",
       "stddev_bias          0.0155863           0.0408741      -1   \n",
       "score                  97.1699             96.6286      -1   \n",
       "\n",
       "                          SibSp             Survived              Ticket  \n",
       "count                       376                  376                 376  \n",
       "mean         0.5159574468085106  0.39095744680851063  262423.08921933087  \n",
       "stddev       1.0977604539171548   0.4886150640852816   454985.4185076666  \n",
       "min                           0                    0              110152  \n",
       "max                           8                    1           WE/P 5735  \n",
       "mean_bias             0.0134805             0.018547          0.00808448  \n",
       "stddev_bias          0.00451871           0.00415668           0.0352492  \n",
       "score                      99.1              98.8648             97.8333  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 查看各属性评估得分\n",
    "evaluation_job_data = submitter.get_evaluation_job_details(job_id=evaluation_job_id)\n",
    "df = evaluation_job_data.to_pandas()\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "97.1617102391623"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "df = df.drop(columns='PassengerId',axis=1)\n",
    "score_list = df.loc['score'].to_list()\n",
    "while -1 in score_list:\n",
    "    score_list.remove(-1)\n",
    "score_list\n",
    "import numpy as np\n",
    "np.mean(score_list)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
